from __future__ import division
from operator import itemgetter, attrgetter

import math
import matplotlib
import os
import pylab
import random
import sys
import time

# step1:
def sampleANDGenerateMachineGeneratedQueryLog():
    # sub_step1: random select the queryIDs from the range [1,10000000]
    randomlySelectedIDList = []
    randomlySelectedIDDict = {}
    
    # in debug
    # totalNumOfRandomlySelectedSamples = 10
    # in production
    totalNumOfRandomlySelectedSamples = 95000
    # for the input file option2
    totalNumOfSamples = 30000000
    # for the input file option1
    # totalNumOfSamples = 10000000
    # assume that the selectedID starts from 0 to 25205179 in the total of 25,205,179 (25M)
    while len(randomlySelectedIDDict) != totalNumOfRandomlySelectedSamples:
        # Return a random integer N such that a <= N <= b
        selectedID = random.randint(1, totalNumOfSamples)
        if selectedID not in randomlySelectedIDDict:
            randomlySelectedIDList.append(selectedID)
            randomlySelectedIDDict[selectedID] = 1
    
    randomlySelectedIDList.sort(cmp=None, key=None, reverse=False)
    # print "randomlySelectedIDList:",randomlySelectedIDList
    print "len(randomlySelectedIDList):",len(randomlySelectedIDList)
    print "len(randomlySelectedIDDict):",len(randomlySelectedIDDict)
    
    # sub_step2:
    # extract those queries from the machine generated query log and form a machine generated query log compared to the human made query log 
    outputFileName = "/data3/obukai/workspace/web-search-engine-wei/polyIRIndexer/95KQueriesMachineGenerated3_20130818_using_new_fake_queryLog"
    outputFileHandler = open(outputFileName,"w")
    
    # option2
    inputFileName = "/data/jrodri04/lm/30M.95KQ.ModKN.txt"
    # option1
    # inputFileName = "/data/jrodri04/lm/10M.95KQ.5gram.ModKN.txt"
    inputFileHandler = open(inputFileName,"r")
    currentLine = inputFileHandler.readline()
    originalLineCounter = 1
    newQueryIDLineCounter = 1
    while currentLine:
        if originalLineCounter in randomlySelectedIDDict:
            # for debug ONLY
            # print "currentLine:",currentLine.strip()
            queryContent = currentLine.strip()
            outputFileHandler.write(str(originalLineCounter) + ":" + str(newQueryIDLineCounter) + ":" + queryContent + "\n")
            newQueryIDLineCounter += 1
        currentLine = inputFileHandler.readline()
        originalLineCounter += 1
    inputFileHandler.close()
    outputFileHandler.close()
    print "inputFileName:",inputFileName
    print "outputFileName:",outputFileName

# step2
def findMachineGeneratedQueriesWhichHaveTheSameSetOfTermsThatHasHighFreqInHumanGeneratedQueryLog():
    # key: NEW queryID
    # value: int NO USE
    alreadyRecordedQueryDict = {}
    outputFileName = "/data3/obukai/workspace/web-search-engine-wei/polyIRIndexer/QueriesWhichContainHighFreqTermAmong95KMachineGenerated3_20130818_using_new_fake_queryLog"
    outputFileHandler = open(outputFileName,"w")
    
    # key: string 
    # value : int (NO USE)
    highFreqTermsInHGQueryLogDict = {}
    inputFileName1 = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/highFreqTermsWithTheirAverageIntersectionSizeAndMetaInfo"
    inputFileHandler = open(inputFileName1,"r")
    for line in inputFileHandler.readlines():
        lineElements = line.strip().split(" ")
        term = lineElements[0]
        if term not in highFreqTermsInHGQueryLogDict:
            highFreqTermsInHGQueryLogDict[term] = 1
        else:
            print "term:",term
            print "duplicate terms"
            exit(1)
    print "len(highFreqTermsInHGQueryLogDict):",len(highFreqTermsInHGQueryLogDict)
    inputFileHandler.close()
    
    # option1
    inputFileName2 = "/data3/obukai/workspace/web-search-engine-wei/polyIRIndexer/95KQueriesMachineGenerated3_20130818_using_new_fake_queryLog"
    # option2
    # inputFileName2 = "/data3/obukai/workspace/web-search-engine-wei/polyIRIndexer/95KQueriesMachineGenerated2"
    inputFileHandler = open(inputFileName2,"r")

    for line in inputFileHandler.readlines():
        elements = line.strip().split(":")
        
        # elements[0]
        newQueryID = int( elements[1] )
        data = elements[2]
        data = data.lower()
        
        for i in range(0,len(data)):
            # print "data[i]:",ord(data[i])
            if not ( (ord(data[i]) >= 48 and ord(data[i]) < 58) or (ord(data[i]) >= 65 and ord(data[i]) < 91) or (ord(data[i]) >= 97 and ord(data[i]) < 123) or (ord(data[i]) == 32) ):
                # Just replace them with a space.
                data = data[:i] + " " + data[i+1:]
        
        queryContent = data
        
        queryContentElements = queryContent.strip().split(" ")
        currentQueryTermDict = {}
        for element in queryContentElements:
            if element.strip() != "":
                if element.strip() not in currentQueryTermDict:
                    currentQueryTermDict[element.strip()] = 1
        
        
        for queryTerm in currentQueryTermDict:
            if queryTerm in highFreqTermsInHGQueryLogDict:
                if newQueryID not in alreadyRecordedQueryDict:
                    outputFileHandler.write(line)
                    alreadyRecordedQueryDict[newQueryID] = 1
            else:
                # this query dies
                pass
    inputFileHandler.close()
    outputFileHandler.close()
    
    print "len(alreadyRecordedQueryDict):",len(alreadyRecordedQueryDict)
    print "inputFileName1:",inputFileName1
    print "inputFileName2:",inputFileName2
    print "outputFileName:",outputFileName
    
    
print "Program Begins..."
# step1:
# sampleANDGenerateMachineGeneratedQueryLog()
# step2:
findMachineGeneratedQueriesWhichHaveTheSameSetOfTermsThatHasHighFreqInHumanGeneratedQueryLog()
print "Program Ends."

